{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a38cb552",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import logging\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13a93c80",
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c23d12bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from docling.backend.pypdfium2_backend import PyPdfiumDocumentBackend\n",
    "from docling.datamodel.base_models import InputFormat\n",
    "from docling.document_converter import (\n",
    "    DocumentConverter,\n",
    "    PdfFormatOption,\n",
    "    WordFormatOption,\n",
    ")\n",
    "from docling.pipeline.simple_pipeline import SimplePipeline\n",
    "from docling.pipeline.standard_pdf_pipeline import StandardPdfPipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fcd4201",
   "metadata": {},
   "outputs": [],
   "source": [
    "_log = logging.getLogger(__name__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d474478",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    input_paths = [\n",
    "        # Path(\"README.md\"),\n",
    "        # Path(\"tests/data/html/wiki_duck.html\"),\n",
    "        # Path(\"tests/data/docx/word_sample.docx\"),\n",
    "        # Path(\"tests/data/docx/lorem_ipsum.docx\"),\n",
    "        # Path(\"tests/data/pptx/powerpoint_sample.pptx\"),\n",
    "        # Path(\"tests/data/2305.03393v1-pg9-img.png\"),\n",
    "        # Path(\"tests/data/pdf/2206.01062.pdf\"),\n",
    "        # Path(\"tests/data/asciidoc/test_01.asciidoc\"),\n",
    "        Path(r\"D:\\YXT_Project\\vector_db_practices\\docling\\10027.pdf\"),\n",
    "    ]\n",
    "\n",
    "    ## for defaults use:\n",
    "    # doc_converter = DocumentConverter()\n",
    "\n",
    "    ## to customize use:\n",
    "\n",
    "    doc_converter = (\n",
    "        DocumentConverter(  # all of the below is optional, has internal defaults.\n",
    "            allowed_formats=[\n",
    "                InputFormat.PDF,\n",
    "                InputFormat.IMAGE,\n",
    "                InputFormat.DOCX,\n",
    "                InputFormat.HTML,\n",
    "                InputFormat.PPTX,\n",
    "                InputFormat.ASCIIDOC,\n",
    "                InputFormat.CSV,\n",
    "                InputFormat.MD,\n",
    "            ],  # whitelist formats, non-matching files are ignored.\n",
    "            format_options={\n",
    "                InputFormat.PDF: PdfFormatOption(\n",
    "                    pipeline_cls=StandardPdfPipeline, backend=PyPdfiumDocumentBackend\n",
    "                ),\n",
    "                InputFormat.DOCX: WordFormatOption(\n",
    "                    pipeline_cls=SimplePipeline  # , backend=MsWordDocumentBackend\n",
    "                ),\n",
    "            },\n",
    "        )\n",
    "    )\n",
    "\n",
    "    conv_results = doc_converter.convert_all(input_paths)\n",
    "\n",
    "    for res in conv_results:\n",
    "        out_path = Path(\"scratch\")\n",
    "        print(\n",
    "            f\"Document {res.input.file.name} converted.\"\n",
    "            f\"\\nSaved markdown output to: {out_path!s}\"\n",
    "        )\n",
    "        _log.debug(res.document._export_to_indented_text(max_text_len=16))\n",
    "        # Export Docling document format to markdowndoc:\n",
    "        with (out_path / f\"{res.input.file.stem}.md\").open(\"w\") as fp:\n",
    "            fp.write(res.document.export_to_markdown())\n",
    "\n",
    "        with (out_path / f\"{res.input.file.stem}.json\").open(\"w\") as fp:\n",
    "            fp.write(json.dumps(res.document.export_to_dict()))\n",
    "\n",
    "        with (out_path / f\"{res.input.file.stem}.yaml\").open(\"w\") as fp:\n",
    "            fp.write(yaml.safe_dump(res.document.export_to_dict()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6366ea1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c839fd6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# 替换为你的 .parquet 文件路径\n",
    "parquet_file_path = r\"D:\\YXT_Project\\vector_db_practices\\docling\\scratch\\multimodal_2025-04-22_165858.parquet\"\n",
    "\n",
    "# 读取文件为 DataFrame\n",
    "df = pd.read_parquet(parquet_file_path)\n",
    "\n",
    "# 显示前几行数据\n",
    "print(df.head())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py310",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
